# -*- coding: utf-8 -*-
# from __future__ import absolute_import
__author__ = 'wwj'
import sys
# reload(sys)
# sys.setdefaultencoding('utf-8')
from celery import Celery
import config
import sys
import os
from celery.utils.log import get_task_logger
import pymongo
from datetime import *
# app = Celery('dispatch',include=['tasks.xvideo'])
app = Celery('dispatch')
app.config_from_object(config)
import hashlib
import requests
import botocore
from subprocess import call
import scrapy
import re
logger = get_task_logger(__name__)
from boto3.session import Session
aws_key = "AKIAJFKTBAZTXUWPGDDA"
aws_secret = "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"
session = Session(aws_access_key_id=aws_key,
aws_secret_access_key=aws_secret,
region_name="us-east-1") # 此处根据自己的 s3 地区位置改变
s3 = session.resource("s3")
client = session.client("s3")
bucket = "xvideoken"
client = s3.Bucket(bucket)

@app.task(bind=True,queue="dmm",max_retries=1,ignore_result=True)
def down(self,url):
    print url
    # if type(urls) == str:
    #     if urls.find(",") > 0:
    #         urls = urls.split(",")
    #     else:
    #         urls = [urls]
    # if type(urls) == str or :
    #     urls = [urls]
    # for url in urls:
    # url = urls
    url = re.sub("\?cache.*", "", url).lstrip("//")
    url = re.sub("\?.*", "", url)
    if url.find("http") < 0:
        url = "http://" + url
    if url.find("https") >= 0:
        url = url.replace("https","http")
    key = hashlib.md5(url.encode("utf8")).hexdigest()
    ext = url.split(".")[-1]
    upload_key = key + "." + ext

    try:
        s3.Object(upload_key).load()
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == "404":
            print url
            response = requests.get(url)
            assert response.status_code == 200
            upload_data = response.content
            file_obj = client.put_object(Key=upload_key,Body=upload_data,
                        Metadata={"fromurl":url},ContentType=response.headers["content-type"])
            print file_obj
            return url
        else:
            raise Exception("gone wrong")
    else:
        print "exits##" + url
        return url

    # objs = list(client.objects.filter(Prefix=upload_key))
    # if len(objs) > 0 and objs[0].key == upload_key:
    #     print "exits##" + url
    #     return url
    # # else:
    #     print url
    #     response = requests.get(url)
    #     assert response.status_code == 200
    #     upload_data = response.content
    #     file_obj = client.put_object(Key=upload_key,Body=upload_data,
    #                 Metadata={"fromurl":url},ContentType=response.headers["content-type"])
    #     print file_obj
    #     return url

